import urllib.request
import ssl
import re
import os


def writeFileBytes(htmlBytes, toPath):
    with open(toPath, "wb")as f:
        f.write(htmlBytes)
def writeFileStr(htmlBytes, toPath):
    with open(toPath, "w") as f:
        f.write(str(htmlBytes))
def getHtmlBytes(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"
    }
    req = urllib.request.Request(url, headers=headers)
    context=ssl._create_unverified_context()
    response = urllib.request.urlopen(req, context=context)
    return response.read()

def qqCrawler(url, toPath):
    htmlBytes = getHtmlBytes(url)

    htmlStr = str(htmlBytes)
'''
    pat = r"[1-9]\d{4,9}"
    re_qq = re.compile(pat)
    qqsList = re_qq.findall(htmlStr)
    # 去重
    qqsList = list(set(qqsList))
    print(qqsList)
    print(len(qqsList))
'''
    pat = '((http|ftp|https)://)(([a-zA-Z0-9\._-]+\.[a-zA-Z]{2,6})|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}))(:[0-9]{1,4})*(/[a-zA-Z0-9\&%_\./-~-]*)?'
    re_url = re.compile(pat)
    urlsList = re_url.findall(htmlStr)
    # 去重
    urlsList = list(set(urlsList))
    print(urlsList)
    print(len(urlsList))

url = "https://www.douban.com/group/topic/110094603/"
toPath = r"F:\qqFile.txt"
qqCrawler(url, toPath)